package com.spider.silence.downloader;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.spider.silence.abstarct.downloader.AbstractCommonDownLoader;
import com.spider.silence.proxy.ProxyPool;
import com.spider.silence.task.SinaNewsModuleTask;
import com.spider.silence.task.manager.CommonTaskManager;
import com.spider.silence.utils.CommonUtils;
import com.spider.silence.utils.DateUtils;
import com.spider.silence.utils.FileUtils;
import com.spider.silence.utils.PropertiesUtil;
import org.apache.log4j.Logger;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.openqa.selenium.Cookie;
import us.codecraft.webmagic.Page;

import java.io.File;
import java.lang.reflect.InvocationTargetException;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * @author Administrator
 * @description 描述
 * @time 2017/3/28 0028
 */
public class SinaNewsDownloader extends AbstractCommonDownLoader implements Cloneable {
    private Logger logger = Logger.getLogger(SinaNewsDownloader.class);
    private String url = "http://news.sina.com.cn/";
    private Map<String,File> files=null;

    public Map<String,File> getFiles(){

        return files;
    }

    public List<Map<String, Object>> getUrl() {
        List<Map<String, Object>> urlList = new ArrayList<Map<String, Object>>();

        Document document = null;
        long future = System.currentTimeMillis() + Long.valueOf(PropertiesUtil.getVal("SinaNews.URLWaitTime"));
        long remianing = Long.valueOf(PropertiesUtil.getVal("SinaNews.URLWaitTime"));
        boolean flag = true;
        while (flag && remianing >= 0) {
            try {
                remianing = future - System.currentTimeMillis();
                document = get(url, true);
                Elements sub_forum = document.select(".cNav2").select("a");

                for (int i = 0; i < sub_forum.size(); i++) {
                    String url = sub_forum.get(i).attr("href");
                    String sub_name = sub_forum.get(i).text();
                    Map<String, Object> map = new HashMap<String, Object>();
                    map.put("url", url);
                    map.put("sub_name", sub_name);
                    urlList.add(map);
                    System.out.println(url + "  " + sub_name);
                }
                flag = false;

          /*  Elements sub_forum = document.select(".main-nav").select("div");
            for(int i = 0;i<sub_forum.size();i++){
                Elements sub_ul = sub_forum.get(i).select("ul");
                for(int j = 0;j<sub_ul.size();j++) {
                    Elements sub_li = sub_ul.get(j).select("li");
                    for(int m = 0;m<sub_li.size();m++){
                        String url = sub_li.get(m).select("a").attr("href");
                        String name = sub_li.get(m).select("a").text();

                        Map<String,Object> map = new HashedMap<String,Object>();
                        map.put("url",url);
                        map.put("sub_name",name);
                        urlList.add(map);
                        System.out.println(url+"  "+name);
                    }
                }

            }*/

            } catch (Exception e) {
                logger.error(e.getMessage(),e);
                e.printStackTrace();
            }
        }


        return urlList;
    }

    public void generateTask(List<Map<String, Object>> urlList) {

        for (int i = 0; i < urlList.size(); i++) {
            try {
                String name = urlList.get(i).get("sub_name").toString();
                String url = urlList.get(i).get("url").toString();

                SinaNewsModuleTask moduleTask = new SinaNewsModuleTask(url, name);
                CommonTaskManager.add("SinaNewsModule", moduleTask);
            }catch (Exception e){
                logger.error(e.getMessage(),e);
                e.printStackTrace();
            }

        }
    }

    public JSONObject crawlItem(String content_url, String channel) {
        files=new ConcurrentHashMap<>();
        JSONObject item = new JSONObject();
        JSONObject data = new JSONObject();

        int page = 1;
        try {
            Document document = null;

            Page pageContent = null;
            try {
                pageContent = getHtml(content_url, true);

                if (pageContent == null && pageContent.getStatusCode() != 200) {
                    pageContent = getHtml(content_url, true);
                }
            } catch (Exception e) {
                logger.error(e.getMessage(),e);
                pageContent = getHtml(content_url, true);
                e.printStackTrace();
            }

            if (null == pageContent) {

            } else if (pageContent.getStatusCode() == 0) {

            } else if (String.valueOf(pageContent.getStatusCode()).startsWith("1")) {

            } else if (pageContent.getStatusCode() == 200) {
                try {
                    document = pageContent.getHtml().getDocument();

                    String news_type = "";
                    if ("国内".equals(channel)) {
                        news_type = "gn";
                    } else if("国际".equals(channel)){
                        news_type = "gj";
                    }else if("社会".equals(channel)){
                        news_type = "sh";
                    }else if("军事".equals(channel)){
                        news_type = "jc";
                    }else if("体育".equals(channel)){
                        news_type = "ty";
                    }else if("科技".equals(channel)){
                        news_type = "kj";
                    }else if("娱乐".equals(channel)){
                        news_type = "yl";
                    }else if("财经".equals(channel)){
                        news_type = "cj";
                    }


                    String id = "";//帖子id
                    String publish_time ="";
                    if("国内".equals(channel) || "国际".equals(channel)||"社会".equals(channel)) {
                        Elements newsElements = document.select("#wrapOuter");
                        String title = newsElements.select(".page-header").select("#artibodyTitle").text();
                        String content = newsElements.select("#articleContent").select(".left").select("#artibody").html();

                        JSONArray img_urls=new JSONArray();
                        Elements imgElements=newsElements.select("#articleContent").select(".left").select("#artibody").select("img[data-mcesrc]");
                        if(imgElements.size()>0){
                            for(int imgIndex=0;imgIndex<imgElements.size();imgIndex++){
                                Element imgElement=imgElements.get(imgIndex);
                                String img_url=imgElement.absUrl("data-mcesrc");
                                img_urls.add(img_url);
                            }
                        }
                        //String replyInfo = newsElements.select("#page-tools").select("#commentCount1").text();
                        //int reply_count = 0;
                        /*if(replyInfo.matches("[0-9]+")){
                            reply_count = Integer.parseInt(replyInfo.split("\"")[1]);
                        }*/

                        String timeInfo = newsElements.select("#page-info").select("#navtimeSource").text();
                        String time = timeInfo.split(" ")[0].trim();

                        publish_time = DateUtils.formatSinaNewsTime(time, channel);



                        // String regex = "[0-9]+(?=[^0-9]*$)";//匹配url中的id
                        String regex = "doc-i[a-z]{1,}[0-9]+(?=[^0-9]*$)";
                        Pattern p = Pattern.compile(regex);
                        Matcher m = p.matcher(content_url);
                        while (m.find()) {
                            id = m.group();
                        }

                        if (id != null && !"".equals(id)) {
                            id = id.split("-")[1];
                            id = id.substring(1);
                        }

                        String spider_time = DateUtils.dateToString(new Date(), "yyyy-MM-dd HH:mm:ss");


                        data.put("id", id);
                        data.put("title", title);
                        data.put("content", FileUtils.filterImg(content));
                        data.put("img_urls",img_urls);
                        data.put("publish_time", publish_time);
                        data.put("spider_time", spider_time);
                        data.put("url", content_url);
                    }else if("军事".equals(channel)||"娱乐".equals(channel)){
                        Elements newsElements = document.select("#pl_main_content");
                        String title = newsElements.select("#main_title").text();

                        String timeInfo = newsElements.select("#page-tools").select(".time-source").select(".titer").text();

                         publish_time = DateUtils.formatSinaNewsTime(timeInfo, channel);

                        String content = newsElements.select("#wrapOuter").select(".content_wrappr_left").select(".artibody").html();
                        JSONArray img_urls=new JSONArray();
                        Elements imgElements=newsElements.select("#wrapOuter").select(".content_wrappr_left").select(".artibody").select("img");
                        if(imgElements.size()>0){
                            for(int imgIndex=0;imgIndex<imgElements.size();imgIndex++){
                                Element imgElement=imgElements.get(imgIndex);
                                String img_url=imgElement.absUrl("src");
                                img_urls.add(img_url);
                            }
                        }
                        String regex = "doc-i[a-z]{1,}[0-9]+(?=[^0-9]*$)";
                        Pattern p = Pattern.compile(regex);
                        Matcher m = p.matcher(content_url);
                        while (m.find()) {
                            id = m.group();
                        }

                        if (id != null && !"".equals(id)) {
                            id = id.split("-")[1];
                            id = id.substring(1);
                        }

                        String spider_time = DateUtils.dateToString(new Date(), "yyyy-MM-dd HH:mm:ss");

                        data.put("id", id);
                        data.put("title", title);
                        data.put("content", FileUtils.filterImg(content));
                        data.put("img_urls",img_urls);
                        data.put("publish_time", publish_time);
                        data.put("spider_time", spider_time);
                        data.put("url", content_url);

                    }else if("体育".equals(channel)){
                        Elements newsElements = document.select(".layout-wrap-a");
                        String title = newsElements.select(".article-a").select("#j_title").text();
                        String timeInfo = newsElements.select(".article-a").select(".article-a__time").text() ;

                        publish_time = DateUtils.formatSinaNewsTime(timeInfo, channel);

                        String content = newsElements.select(".article-a").select("#artibody").html();
                        JSONArray img_urls=new JSONArray();
                        Elements imgElements=newsElements.select(".article-a").select("#artibody").select("img");
                        if(imgElements.size()>0){
                            for(int imgIndex=0;imgIndex<imgElements.size();imgIndex++){
                                Element imgElement=imgElements.get(imgIndex);
                                String img_url=imgElement.absUrl("src");
                                img_urls.add(img_url);
                            }
                        }
                        String regex = "doc-i[a-z]{1,}[0-9]+(?=[^0-9]*$)";
                        Pattern p = Pattern.compile(regex);
                        Matcher m = p.matcher(content_url);
                        while (m.find()) {
                            id = m.group();
                        }

                        if (id != null && !"".equals(id)) {
                            id = id.split("-")[1];
                            id = id.substring(1);
                        }

                        String spider_time = DateUtils.dateToString(new Date(), "yyyy-MM-dd HH:mm:ss");

                        data.put("id", id);
                        data.put("title", title);
                        data.put("content", FileUtils.filterImg(content));
                        data.put("img_urls",img_urls);
                        data.put("publish_time", publish_time);
                        data.put("spider_time", spider_time);
                        data.put("url", content_url);
                    }else if("财经".equals(channel)){
                        Elements newsElements = document.select("#wrapOuter");
                        String title = newsElements.select(".page-header").select("#artibodyTitle").text();
                        String content = newsElements.select("#articleContent").select(".left").select("#artibody").html();
                        JSONArray img_urls=new JSONArray();
                        Elements imgElements=newsElements.select("#articleContent").select(".left").select("#artibody").select("img");
                        if(imgElements.size()>0){
                            for(int imgIndex=0;imgIndex<imgElements.size();imgIndex++){
                                Element imgElement=imgElements.get(imgIndex);
                                String img_url=imgElement.absUrl("src");
                                img_urls.add(img_url);
                            }
                        }
                        //String replyInfo = newsElements.select("#page-tools").select("#commentCount1").text();
                        //int reply_count = 0;
                        /*if(replyInfo.matches("[0-9]+")){
                            reply_count = Integer.parseInt(replyInfo.split("\"")[1]);
                        }*/

                        String timeInfo = newsElements.select(".page-info").select(".time-source").text();
                        String time = timeInfo.split(" ")[0].trim();

                        publish_time = DateUtils.formatSinaNewsTime(time, channel);



                        // String regex = "[0-9]+(?=[^0-9]*$)";//匹配url中的id
                        String regex = "doc-i[a-z]{1,}[0-9]+(?=[^0-9]*$)";
                        Pattern p = Pattern.compile(regex);
                        Matcher m = p.matcher(content_url);
                        while (m.find()) {
                            id = m.group();
                        }

                        if (id != null && !"".equals(id)) {
                            id = id.split("-")[1];
                            id = id.substring(1);
                        }

                        String spider_time = DateUtils.dateToString(new Date(), "yyyy-MM-dd HH:mm:ss");


                        data.put("id", id);
                        data.put("title", title);
                        data.put("content", FileUtils.filterImg(content));
                        data.put("img_urls",img_urls);
                        data.put("publish_time", publish_time);
                        data.put("spider_time", spider_time);
                        data.put("url", content_url);

                    }else if("科技".equals(channel)){
                        Elements newsElements = document.select("#pl_main_content");
                        if(newsElements.size()==0||newsElements == null){
                            newsElements = document.select(".wrap").select(".main_left");


                            String timeInfo = newsElements.select(".artInfo").select("#pub_date").text();
                            SimpleDateFormat sdf = new SimpleDateFormat("yyyy年MM月dd日hh:mm");

                            if(timeInfo==null||"".equals(timeInfo)){
                                newsElements = document.select(".wrap").select(".blkContainer");
                                timeInfo = newsElements.select(".artInfo").select("#pub_date").text().trim();
                                sdf = new SimpleDateFormat("yyyy年MM月dd日 hh:mm");
                            }

                            String title = newsElements.select("#artibodyTitle").text();
                            String content = newsElements.select("#artibody").html();
                            JSONArray img_urls=new JSONArray();
                            Elements imgElements=newsElements.select("#artibody").select("img");
                            if(imgElements.size()>0){
                                for(int imgIndex=0;imgIndex<imgElements.size();imgIndex++){
                                    Element imgElement=imgElements.get(imgIndex);
                                    String img_url=imgElement.absUrl("src");
                                    img_urls.add(img_url);
                                }
                            }
                            SimpleDateFormat sdf1 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");



                             publish_time = "";
                            try {
                                Date date = sdf.parse(timeInfo);
                                publish_time  =  sdf1.format(date);
                            }catch (Exception e){
                                logger.error(e.getMessage(),e);
                                e.printStackTrace();
                            }


                            String regex = "doc-i[a-z]{1,}[0-9]+(?=[^0-9]*$)";
                            Pattern p = Pattern.compile(regex);
                            Matcher m = p.matcher(content_url);
                            if (m.find()) {
                                id = m.group();
                                if (id != null && !"".equals(id)) {
                                    id = id.split("-")[1];
                                    id = id.substring(1);
                                }
                            }else{
                                String regex1 = "pid_[0-9]+(?=[^0-9]*$)";
                                Pattern p1 = Pattern.compile(regex1);
                                Matcher m1 = p1.matcher(content_url);
                                if(m1.find()){
                                    id = m1.group();
                                    id = id.split("_")[1];
                                }

                            }


                            String spider_time = DateUtils.dateToString(new Date(), "yyyy-MM-dd HH:mm:ss");

                            data.put("id", id);
                            data.put("title", title);
                            data.put("content", FileUtils.filterImg(content));
                            data.put("img_urls",img_urls);
                            data.put("publish_time", publish_time);
                            data.put("spider_time", spider_time);
                            data.put("url", content_url);


                        }else{
                            String title = newsElements.select("#main_title").text();

                            String timeInfo = newsElements.select("#page-tools").select(".time-source").select(".titer").text();

                            publish_time = DateUtils.formatSinaNewsTime(timeInfo, channel);

                            String content = newsElements.select("#wrapOuter").select(".content_wrappr_left").html();

                            if(newsElements.select("#wrapOuter").select(".content_wrappr_left").select("#artibody")!=null){
                                content = newsElements.select("#wrapOuter").select(".content_wrappr_left").select("#artibody").html();
                            }
                            JSONArray img_urls=new JSONArray();
                            Elements imgElements=newsElements.select("#wrapOuter").select(".content_wrappr_left").select("img");
                            if(imgElements.size()>0){
                                for(int imgIndex=0;imgIndex<imgElements.size();imgIndex++){
                                    Element imgElement=imgElements.get(imgIndex);
                                    String img_url=imgElement.absUrl("src");
                                    img_urls.add(img_url);
                                }
                            }
                            String regex = "doc-i[a-z]{1,}[0-9]+(?=[^0-9]*$)";
                            Pattern p = Pattern.compile(regex);
                            Matcher m = p.matcher(content_url);
                            while (m.find()) {
                                id = m.group();
                            }

                            if (id != null && !"".equals(id)) {
                                id = id.split("-")[1];
                                id = id.substring(1);
                            }

                            String spider_time = DateUtils.dateToString(new Date(), "yyyy-MM-dd HH:mm:ss");

                            data.put("id", id);
                            data.put("title", title);
                            data.put("content", FileUtils.filterImg(content));
                            data.put("img_urls",img_urls);
                            data.put("publish_time", publish_time);
                            data.put("spider_time", spider_time);
                            data.put("url", content_url);
                        }

                    }

                    long tsp= DateUtils.dateToTimestamp( DateUtils.parseDate(publish_time)).getTime();
                    String filePrefix="SinaNews";//文件前缀
                    String fileName=filePrefix+"_"+tsp+"_"+"_"+id;
                    String publishDate=DateUtils.formatDate(DateUtils.parseDate(publish_time)).replace("-","-");
                    String speechesId="SinaNews_speeches_"+publishDate+"_"+fileName;
                    File file= FileUtils.createTempFile(speechesId+"_pageNo_1",document.html().toString());
                    files.put(speechesId+"_pageNo_1",file);
                    //帖子首页的文件


                    // Elements replyInfos = document.select(".reply-box");

                    int reply_pageSize = 20;
                    int reply_page = 1;

                    JSONArray reply_nodes = new JSONArray();
                    List<Map<String, Object>> list = new ArrayList<Map<String, Object>>();
                    int reply_count = 0;
                    /**
                     * 根据第一页内容查出回复数和总页数
                     */
                    if (id != null && !"".equals(id)) {
                        String newsId = "";
                        if(id.matches("[0-9]+")&&"科技".equals(channel)){
                            newsId = "dtech-c-p-"+id;
                        }else{
                            newsId = "comos-" + id;
                        }
                        String reply_url = "http://comment5.news.sina.com.cn/page/info?format=js&channel=" + news_type + "&newsid=" + newsId + "&group=0&compress=1&ie=gbk&oe=gbk&page=1&page_size=" + reply_pageSize + "&jsvar=requestId_23827117";
                        Document reply_document = null;
                        Page reply_pageContent = null;

                        try {
                            reply_pageContent = getHtml(reply_url);
                            if (reply_pageContent == null && reply_pageContent.getStatusCode() != 200) {
                                reply_pageContent = getHtml(reply_url, true);
                            }
                        } catch (Exception e) {
                            logger.error(e.getMessage(),e);
                            reply_pageContent = getHtml(reply_url, true);
                            e.printStackTrace();
                        }

                        if (null == reply_pageContent) {

                        } else if (reply_pageContent.getStatusCode() == 0) {

                        } else if (String.valueOf(reply_pageContent.getStatusCode()).startsWith("1")) {

                        } else if (reply_pageContent.getStatusCode() == 200) {
                            reply_document = reply_pageContent.getHtml().getDocument();
                            String reply_result = reply_document.select("body").html();
                            int begin = reply_result.indexOf("=");
                            String new_reply_result = reply_result.substring(begin + 1);
                            JSONObject reply_object = JSONObject.parseObject(new_reply_result);

                            if (reply_object.getJSONObject("result").containsKey("count")) {
                                JSONObject countInfo = reply_object.getJSONObject("result").getJSONObject("count");
                                reply_count = countInfo.getInteger("show");
                                reply_page = (reply_count - 1) / reply_pageSize + 1;
                                if (reply_count > 0) {
                                    JSONArray replyArray = reply_object.getJSONObject("result").getJSONArray("cmntlist");
                                    for(int i = 0;i<replyArray.size();i++) {
                                        JSONObject new_reply = new JSONObject();

                                        JSONObject replyInfo = replyArray.getJSONObject(i);

                                        new_reply.put("ancestor_id", id);
                                        new_reply.put("parent_id", replyInfo.get("parent"));
                                        new_reply.put("dislike_count", replyInfo.get("against"));
                                        new_reply.put("like_count", replyInfo.get("agree"));
                                        new_reply.put("id", replyInfo.get("mid"));
                                        new_reply.put("spider_time", DateUtils.dateToString(new Date(), "yyyy-MM-dd HH:mm:ss"));
                                        new_reply.put("publish_time", replyInfo.get("time"));
                                        new_reply.put("publish_user_id", replyInfo.get("uid"));
                                        new_reply.put("publish_user", replyInfo.get("nick"));
                                        new_reply.put("content", FileUtils.filterImg(replyInfo.get("content").toString()));
                                        String publish_user_photo = "http://tp1.sinaimg.cn/" + replyInfo.get("uid") + "/50/1";

                                        new_reply.put("publish_user_photo", publish_user_photo);

                                        reply_nodes.add(new_reply);
                                    }
                                }


                            }

                            JSONArray replys = new JSONArray();
                            if(reply_page>=2){
                                int threadNum = Integer.valueOf(PropertiesUtil.getVal("SinaNews.ReplyThreadCount"));
                                ExecutorService executorService = Executors.newFixedThreadPool(threadNum);
                                try {
                                    List<FutureTask<JSONArray>> futureTasks = new ArrayList<FutureTask<JSONArray>>();
                                    ArrayList<ArrayList<Integer>> splitPages = CommonUtils.splitPages(2, reply_page, threadNum);
                                    for (int i = 0; i < splitPages.size(); i++) {
                                        SinaNewsDownloader downLoader = this.cloneDownLoader();
                                        SinaNewsDownloader.ReplyCrawler callable = new SinaNewsDownloader.ReplyCrawler(downLoader, splitPages.get(i), id, reply_pageSize, channel,speechesId);
                                        FutureTask<JSONArray> futureTask = new FutureTask<JSONArray>(callable);
                                        futureTasks.add(futureTask);
                                        executorService.submit(futureTask);
                                    }

                                    for (FutureTask<JSONArray> futureTask : futureTasks) {
                                        JSONArray replyArrays = futureTask.get();
                                        for (int i = 0; i < replyArrays.size(); i++) {
                                            JSONObject object = replyArrays.getJSONObject(i);
                                            replys.add(object);
                                        }
                                    }
                                    if (replys.size() > 0 && replys != null) {
                                        for (int i = 0; i < replys.size(); i++) {
                                            JSONObject reply_object1 = replys.getJSONObject(i);
                                            reply_nodes.add(reply_object1);
                                        }
                                    }
                                    if (reply_nodes.size() > 0 && reply_nodes != null) {
                                        for (int i = 0; i < reply_nodes.size(); i++) {
                                            JSONObject reply_object2 = reply_nodes.getJSONObject(i);
                                            Map<String, Object> map = reply_object2;
                                            list.add(map);
                                        }
                                    }
                                    Collections.sort(list, new Comparator<Map<String, Object>>() {

                                        public int compare(Map<String, Object> o1, Map<String, Object> o2) {
                                            String s1 = o1.get("publish_time").toString();
                                            String s2 = o2.get("publish_time").toString();
                                            int flag = s2.compareTo(s1);
                                            return flag;
                                        }
                                    });

                                } catch (Exception e) {
                                    e.printStackTrace();
                                    logger.error(e.getMessage(), e);
                                } finally {
                                    executorService.shutdown();
                                }
                            }else{
                                if (reply_nodes.size() > 0 && reply_nodes != null) {
                                    for (int i = 0; i < reply_nodes.size(); i++) {
                                        JSONObject new_reply_object = reply_nodes.getJSONObject(i);
                                        Map<String, Object> map = new_reply_object;
                                        list.add(map);
                                    }
                                }
                            }

                        }


                    }else{

                    }
                    data.put("reply_count",reply_count);
                    data.put("reply_nodes",list);



                } catch (Exception e) {
                    logger.error(e.getMessage(),e);
                    e.printStackTrace();
                    System.out.println(channel+":"+content_url+"解析出错");
                }
            } else if (String.valueOf(pageContent.getStatusCode()).startsWith("3")) {

            } else if (String.valueOf(pageContent.getStatusCode()).startsWith("4")) {

            } else if (String.valueOf(pageContent.getStatusCode()).startsWith("5")) {

            } else {

            }

        } catch (Exception e) {
            logger.error(e.getMessage(),e);
            e.printStackTrace();
            System.out.println(channel+":"+content_url+"解析出错");
        }
        item.put("data", data);
        return item;
    }


    public SinaNewsDownloader cloneDownLoader() throws InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException {
        SinaNewsDownloader loader = new SinaNewsDownloader();
        loader.cookies = (HashSet<Cookie>) this.cookies.clone();
        loader.headers = (HashMap<String, String>) this.headers.clone();
        return loader;
    }

    public static void main(String[] args) {
        SinaNewsDownloader downloader = new SinaNewsDownloader();
       /* List<Map<String, Object>> urlList = downloader.getUrl();
        downloader.generateTask(urlList);*/

        JSONObject object = downloader.crawlItem("http://tech.sina.com.cn/i/2017-07-07/doc-ifyhwehx5332560.shtml","科技");
        System.out.println(object.toString());

        //downloader.test();


       /*String url = "http://api.roll.news.sina.com.cn/zt_list?channel=news&cat_1=gnxw&cat_2=gdxw1&level==1||=2&show_ext=1&show_all=1&show_num=22&tag=1&format=json&page=4&callback=newsloadercallback&_=1490858104999";
        //url = url.replaceAll("=","%3D");
        url = url.replaceAll("\\|","%124");
        System.out.println(url);*/
    }



    public class ReplyCrawler implements Callable<JSONArray>{
        private SinaNewsDownloader downloader;
        private ArrayList<Integer> pages;
        private String channel;
        private String ancestor_id;
        private int pageSize;
        private String speeches_id;

        @Override
        public JSONArray call() throws Exception {
            JSONArray array =new JSONArray();
            String news_type = "";
            if("国内".equals(channel)) {
                news_type = "gn";
            }else if("国际".equals(channel)){
                news_type = "gj";
            }else if("社会".equals(channel)){
                news_type = "sh";
            }else if("军事".equals(channel)){
                news_type = "jc";
            }else if("体育".equals(channel)){
                news_type = "ty";
            }else if("科技".equals(channel)){
                news_type = "kj";
            }else if("娱乐".equals(channel)){
                news_type = "yl";
            }else if("财经".equals(channel)){
                news_type = "cj";
            }
            for(int k=0;k<pages.size();k++){
                String new_url = "http://comment5.news.sina.com.cn/page/info?format=js&channel="+news_type+"&newsid=comos-"+ancestor_id+"&group=0&compress=1&ie=gbk&oe=gbk&page="+pages.get(k)+"&page_size="+pageSize+"&jsvar=requestId_23827117";
                Document document = null;
                Page pageContent = null;

                try {
                    pageContent = downloader.getHtml(new_url);

                    if (pageContent == null && pageContent.getStatusCode() != 200) {
                        pageContent = downloader.getHtml(new_url, true);
                    }
                }catch (Exception e){
                    pageContent = downloader.getHtml(new_url, true);
                    e.printStackTrace();
                }

                if (null == pageContent) {

                } else if (pageContent.getStatusCode() == 0) {

                } else if (String.valueOf(pageContent.getStatusCode()).startsWith("1")) {

                } else if (pageContent.getStatusCode() == 200) {
                    try {
                        document = pageContent.getHtml().getDocument();
                        File file=FileUtils.createTempFile(speeches_id+"_pageNo_"+pages.get(k),document.html().toString());
                        files.put(speeches_id+"_pageNo_"+pages.get(k),file);
                        String reply_result = document.select("body").html();
                        int begin = reply_result.indexOf("=");
                        String new_reply_result = reply_result.substring(begin + 1);
                        JSONObject reply_object = null;
                        try {
                            reply_object = JSONObject.parseObject(new_reply_result);
                        } catch (Exception e) {
                            e.printStackTrace();
                            System.out.println(reply_object);
                        }
                        if (reply_object.getJSONObject("result").containsKey("count")) {
                            JSONObject countInfo = reply_object.getJSONObject("result").getJSONObject("count");
                            int reply_count = countInfo.getInteger("show");
                            //reply_page = (reply_count - 1) /reply_pageSize + 1;
                            if (reply_count > 0) {
                                JSONArray replyArray = reply_object.getJSONObject("result").getJSONArray("cmntlist");
                                for (int i = 0; i < replyArray.size(); i++) {
                                    JSONObject new_reply = new JSONObject();

                                    JSONObject replyInfo = replyArray.getJSONObject(i);

                                    new_reply.put("ancestor_id", ancestor_id);
                                    new_reply.put("parent_id", replyInfo.get("parent"));
                                    new_reply.put("dislike_count", replyInfo.get("against"));
                                    new_reply.put("like_count", replyInfo.get("agree"));
                                    new_reply.put("id", replyInfo.get("mid"));
                                    new_reply.put("spider_time", DateUtils.dateToString(new Date(), "yyyy-MM-dd HH:mm:ss"));
                                    new_reply.put("publish_time", replyInfo.get("time"));
                                    new_reply.put("publish_user_id", replyInfo.get("uid"));
                                    new_reply.put("publish_user", replyInfo.get("nick"));
                                    new_reply.put("content", FileUtils.filterImg(replyInfo.get("content").toString()));
                                    String publish_user_photo = "http://tp1.sinaimg.cn/" + replyInfo.get("uid") + "/50/1";

                                    new_reply.put("publish_user_photo", publish_user_photo);

                                    array.add(new_reply);

                                }
                            }
                        }
                    }catch (Exception e){
                        e.printStackTrace();
                        continue;
                    }
                }else{

                }
            }



            return array;
        }

        public ReplyCrawler(SinaNewsDownloader downloader, ArrayList<Integer> pages, String ancestor_id, int pageSize, String channel,String speeches_id){
                this.downloader = downloader;
                this.pages = pages;
                this.ancestor_id = ancestor_id;
                this.pageSize = pageSize;
                this.channel = channel;
                this.speeches_id=speeches_id;

        }



    }

}



